package ar.unc.famaf.pln.giveselector.loganalyser;

import java.io.*;
import java.util.*;
import java.util.zip.GZIPInputStream;

import give.comm.*;
import give.formula.*;
import give.mapreplay.GameEvent;
import give.nlgserver.discretize.RegionDiscretizer;
import give.nlgserver.planning.LazyFfPlannerInterface;
import give.nlgserver.planning.PlannerInterface;
import give.world.Action;
import give.world.World;

import ar.unc.famaf.pln.giveselector.SelectorGiveServer;
import ar.unc.famaf.pln.giveselector.builders.ACLBuilder;
import ar.unc.famaf.pln.giveselector.classifier.*;
import ar.unc.famaf.pln.giveselector.context.*;
import ar.unc.famaf.pln.giveselector.pickers.*;
import ar.unc.famaf.pln.giveselector.selectors.*;
import ar.unc.famaf.pln.giveselector.utils.GIVEConnector;

/**
 * This class is intended to simulate the CL behavior for already played GIVE games.
 * It takes a GIVE log as an input and determines which annotated Utterance object 
 * from the H-H corpus is associated with a specific logged instruction from the given 
 * H-M corpus.
 * The CL system logs the success or failure of an utterance depending
 * on whether the DF's action is contained in the utterance reaction.
 * The way CL implemented this was not quite right because the manipulate/take
 * kind of actions were not logged at all (for example).
 * To get useful information about how the player reacted to an instruction occurrence 
 * we need to recreate/simulate the GIVE2.5 evaluation games trough the logs and then
 * save the necessary information and associate it with each Utterance.
 *
 * @author David Racca ( david.racca@gmail.com )
 *
 */
public class LogAnalyser
{
	protected GIVEConnector connector;
	protected PlannerInterface planner;
	protected RegionDiscretizer discretizer; 

	// The Name of the NLG Server as figures in the database
	protected static final String NLG_SERVER_NAME = "cl";

	// The MySql Database information (URL, username, passw) of the input H-M corpus 
	// should be in the following file
	protected static final String MATCHMAKER_FILE = "datas/loganalyser-match-config.xml";

	// The first input. The annotated utterances from the H-H corpus that were used in the evaluation
	// This file will be regenerated and the new one will contain data about DF's performance during the evaluation
	protected static final String OLD_ANNOTATION_FILE = "datas/action-utterances-with-id.bin.gz";

	// The output. It will content the serialization of the new 'utterances' object
	// The new utterances object will contain the DF's performance information (Associated Occurrence objects)
	protected static final String NEW_ANNOTATION_FILE = "datas/action-utterances-with-id-performance.bin.gz";

	// Change this if you want to redirect the standard output to a file. Select the name of the output file
	// If you are using eclipse, you can do the same by configuring the "Run Configurations..."
	protected static final boolean REDIRECT_STD_OUTPUT = false;
	protected static final String STD_OUTPUT_FILE = "datas/OUTPUT_RECONSTRUCTOR.DAT";

	// <world name, utterances> The Annotated Utterances from the H-H corpus that 
	// will be modified to contain performance information of each analyzed log instruction
	protected Map<String, Set<Utterance>> utterances = new HashMap<String, Set<Utterance>>();

	protected static Integer nUtts = 0;

	// Number of times an instruction from the log matched with exactly one Utterance 
	protected static Integer nMatchUnique = 0;

	// Number of times an instruction from the log matched with more than one Utterance
	protected static Integer nMultipleMatch = 0;

	// Number of times an instruction from the log did not match with any Utterance
	protected static Integer nNoMatch = 0;

	// Number of non matching utterances which are hard-coded instructions
	protected static Integer nHardcoded = 0;

	// Number of non matching utterances which has not been annotated before 
	// This includes also the non matching utterances that are in the H-H corpus
	// but cannot be selected by the selector (GIVE's tutorial ones)
	protected static Integer nNotAnnotated = 0;

	private IPicker picker;
	private ISelector selector;


	/**
	 * Prints information about the number of utterances that were analyzed, 
	 * and the number of them that matched with one or multiple annotated utterance.
	 */
	public void printStatistics()
	{
		System.out.println("\n\n==============================\n");
		System.out.println("========= STATISTICS =========\n");
		System.out.println("==============================\n");

		System.out.println("Num of Analysed Utterances: "+ nUtts + " (100%)");
		System.out.println("Num of Match Unique: "+ nMatchUnique
			+ " (" + percentage(nMatchUnique, nUtts) + "%)");

		System.out.println("Num of multiple match: "+ nMultipleMatch
			+ " (" + percentage(nMultipleMatch, nUtts) + "%)");

		System.out.println("Num of No match: "+ nNoMatch
			+ " (" + percentage(nNoMatch, nUtts) + "%)");

		System.out.println("Num of hard-coded: "+ nHardcoded
			+ " (" + percentage(nHardcoded, nUtts) + "%)");

		System.out.println("Num of not in H-H corpus or with empty Reaction: "+ nNotAnnotated
				+ " (" + percentage(nNotAnnotated, nUtts) + "%)");
	}


	/**
	 * @return (success / total) * 100 
	 */
	public static float percentage(int success, int total)
	{
		return (total!=0) ? (((float) success) / ((float) total) * 100.0f) : 0.f;
	}


	/**
	 * Runs the LogAnalyser over the specified GIVE log database. The database info
	 * is contained in the MATCHMAKER_FILE. 
	 * The LogAnalyser will get data from the GIVE log, matching each 
	 * instruction occurrence in the log with the associated
	 * utterance in the database. It will create UtteranceP objects that have 
	 * the performance information about an Utterance. After that, it will save 
	 * the utterances list of UtteranceP objects in the NEW_ANNOTATION_FILE replacing
	 * the old OLD_ANNOTATION_FILE with this new one.
	 * @param args
	 * @throws IOException
	 */
	public static void main(String[] args) throws IOException
	{
		//redirecting standard output to a file
		if (REDIRECT_STD_OUTPUT)
			System.setOut(
				new PrintStream(
					new BufferedOutputStream(
						new FileOutputStream(STD_OUTPUT_FILE))));
	
		LogAnalyser reconstructor = new LogAnalyser();
		reconstructor.printStatistics();
	}


	/**
	 * Creates a LogAnalyser object 
	 */
	public LogAnalyser()
	{
		connector = new GIVEConnector(MATCHMAKER_FILE);
		picker = new BasicPicker();
		selector = new ACLSelector(new File(OLD_ANNOTATION_FILE));
		utterances = LogAnalyser.load(new File(OLD_ANNOTATION_FILE));

		List<Integer> ids = connector.getGameIdsFromServer(NLG_SERVER_NAME);
		System.out.println("\n Valid Games: " + ids);

		for (int i=0; i<ids.size(); i++)
		{
			int gameId = ids.get(i);
			System.out.println("\n\n\nProcessing game " + gameId + " (" + (i+1) + "/" + ids.size() + ")\n\n");
			World world = connector.getWorld(gameId);
			planner = new LazyFfPlannerInterface(new RegionDiscretizer(world));
			discretizer = new RegionDiscretizer(world);
			analyseLog(world, gameId);
			ACLBuilder.save(new File(NEW_ANNOTATION_FILE), utterances);
		}

	}


	/**
	 * @return the utterances object (It's called in the main method)
	 * about each utterance
	 */
	public Map<String, Set<Utterance>> getUtterances() {
		return utterances;
	}
	
	
	/**
	 * @param selected
	 * @param text
	 * @return The SubSet of the given Utterances which text content 
	 * is equal to the given text 
	 */
	protected Set<Utterance> matchByText(Set<Utterance> selected, String text)
	{
		Iterator<Utterance> it = selected.iterator();
		Set<Utterance> candidates = new HashSet<Utterance>();
		while (it.hasNext())
		{
			Utterance u = it.next();
			if (u.getText().contentEquals(text))
			{
				candidates.add(u);
			}
		}
		return candidates;
	}


	/**
	 * @return All the hard-coded utterances generated by CL
	 */
	public static List<String> getCLStaticMessages()
	{
		List<String> result = new ArrayList<String>();
		result.add("Loading...");
		result.add("we are too close to alarm, get a bit further");
		result.add("slowly, we are too close to alarm");
		result.add("click and win :)");
		result.add("yup, one button done :D");
		result.add("2 buttons done ;)");
		result.add("oops! that click reset part of the security");
		result.add("but we still can get to the trophy :)");
		result.add("it's crucial to press buttons only once");
		result.add("otherwise some button effects are undone");
		result.add("ups, that click wasn't in the sequence");
		result.add("luckily it was harmless");
		result.add("go");
		result.addAll(SelectorGiveServer.AlarmGroundingUtterances());
		result.add("Hi! that was the tutorial");
		result.add("I'm your game partner");
		result.add("Our goal is to steal a golden trophy");
		result.add("without activating alarms");
		result.add("I am at the house's control room");
		result.add("where I can monitor the security system");
		result.add("To disable security and get the trophy");
		result.add("we need to press the right buttons");
		result.add("See the numbers below, on the right?");
		result.add("they'll be updated as we advance");
		result.add("let's go for the trophy!");
		return result;
	}


	/**
	 * Deletes the progress information that CL has added at the end of each instruction
	 * and returns the resulting cleaned message.
	 */
	public static String cleanInstruction(String text)
	{
		String result = null;
		if (text!=null)
		{
			int i = text.indexOf("\n");
			if (i>0)
			{
				result = (text.isEmpty()) ? "" : text.substring(0, i);
			}
			else
				result = text;
		}
		return result;
	}


	/**
	 * @param text
	 * @return True if the given text does not belong to the annotated
	 * set of utterances because was generated by "hand-rules"
	 */
	public boolean isGeneratedByHand(String text, List<String> hardCoded)
	{
		return (hardCoded.contains(text)
			|| text.matches("^They show we pressed \\d+ out of \\d+ buttons$")
			|| text.matches("^\\d+ done :\\)$")
			|| text.matches("^If we don't make mistakes we'll press \\d+$"));
	}


	/**
	 * @param text
	 * @param utts
	 * @return True if there is an Utterance in utts that
	 * has the given text with empty discretized action.
	 */
	public boolean isAnnotatedNonEmptyReaction(String text, Set<Utterance> utts)
	{
		Iterator<Utterance> it = utts.iterator();
		while (it.hasNext())
		{
			Utterance utt = it.next();
			if (utt.getText().equals(text) 
					&& !utt.getResponse().getDiscretizedActions().isEmpty())
				return true;
		} 
		return false;
	}


	/**
	 * Creates an UtteranceP object that is equal to the given Utterance object,
	 * but includes an Occurrence object. Then it adds the resulting object into 
	 * the Utterances Sets.
	 */
	public static void addUtteranceP(Utterance utt, Integer gameId, Date timestamp, 
			Response reaction, Set<Utterance> utts, List<Occurrence> relatedOccurrences) 
	{
		UtteranceP uttp = null;
		Utterance utterance = null;
		Iterator<Utterance> it = utts.iterator();
		while (it.hasNext())
		{
			utterance = it.next();
			if (utterance.getId() == utt.getId()) break;
		}

		if (utterance instanceof UtteranceP)
		{
			uttp = (UtteranceP) utterance;
		}
		else
		{
			uttp = new UtteranceP(utterance);
		}

		Occurrence occurrence = new Occurrence(uttp, gameId, timestamp, reaction);
		System.out.println("Occurrence: " + occurrence);
		uttp.addOccurrence(occurrence);
		relatedOccurrences.add(occurrence);
		System.out.println("Related Occurrences: " + relatedOccurrences);
		utts.remove(utterance);
		utts.add(uttp);
	}


	/**
	 * It uses the selector object with the initial state world and the old context
	 * to get a the set of annotated Utterances which are the candidates to match with.
	 * Then it compares lexically the given text with each of the candidates to filter
	 * the candidates list. After this, it computes the DF's discretized reaction by using the
	 * ACLBuilder.computeDiscretizedResponse() method by passing the initialState world
	 * as the initial state and the world as the goal state.
	 * Finally it adds this instruction occurrence to the UtteranceP object and prints
	 * some useful debug information.
	 * @return False if the planner cannot compute a plan for any initialStates object
	 * 
	 */
	public boolean processStatusTextMessage(String text, List<World> initialStates, World world, 
			Response reaction, Integer gameId, Date timestamp, List<Occurrence> relatedOccurrences)
	{
		System.out.println("\n\n\n======================================================================\n");
		System.out.println("Verbalized Instruction: " + text);
		System.out.println("Time: " + timestamp + ":" + timestamp.getTime() + "\n");

		// Uses several past initial states to do the matching
		boolean matched = false;
		for (int i=initialStates.size()-1; i>=0 && !matched; i--)
		{
			// Recreates the same context object that CL used
			World initialState = initialStates.get(i);
			List<Atom> plan = getOptimizedPlan(planner, initialState, initialState.getGoals());
			if (plan == null || plan.isEmpty())
			{
				// if we cannot find a plan we cannot use that game
				System.out.println("[Error]: null plan for '" + text + "' ??");
				return false;
			}
			Context context = new Context(ACLBuilder.getPlayerRegion(discretizer, initialState));
			context.setFutureActions(plan);
			context.setOrientation(initialState.getPlayerOrientation());

			// Uses the Selector to reproduce the set of candidates for this context
			Set<Utterance> selected = selector.selectUtterances(world.getName(), context);

			// Checks which ones of the candidates match with text
			Set<Utterance> candidates = matchByText(selected, text);

			System.out.println("\nCandidates: " + candidates);

			if (!candidates.isEmpty())
			{
				Utterance candidate = picker.pick(candidates, context);
				System.out.println("Picked: " + candidate);
				addUtteranceP(candidate, gameId, timestamp, reaction, utterances.get(world.getName()), relatedOccurrences);
				nMatchUnique += (candidates.size()==1) ? 1:0;
				nMultipleMatch += (candidates.size()>1) ? 1:0;
				matched = true;
			}
		}
		
		if (!matched)
		{
			System.out.println("|||||| NOT Matched ||||||");
			++nNoMatch;
		}
		
		System.out.println("DF's Discretized Reaction: " + reaction.getDiscretizedActions());
		System.out.println("DF's Executed    Reaction: " + reaction.getExecutedActions());
		System.out.println("======================================================================\n\n");

		return true;
	}


	/**
	 * Creates a new world that's a copy of the given world and
	 * adds it to the initialStates list of world objects. 
	 */
	public void addNewWorld(World world, List<World> initialStates)
	{
		if (world!=null)
		{
			World initial = new World();
			world.copyTo(initial);
			initialStates.add(initial);
		}
	}

	
	/**
	 * Check if the given text is a hardCoded instruction generated by the CL
	 * system or if it is not annotated or it is annotated but with an empty 
	 * DF's reaction.
	 * @return True if the LogReconstructor should omit the given text instruction
	 * or when text is equal to null.
	 */
	protected boolean shouldBeFiltered(String text, List<String> hardCoded, Date timestamp, String worldName)
	{
		if (text == null)
			return true;

		if (isGeneratedByHand(text, hardCoded))
		{
			++nHardcoded;
			System.out.println("\n[---Filtering HardCoded---]: " + text);
			System.out.println("[Time]: " + timestamp + ":" + timestamp.getTime() + "\n");
			return true;
		}

		if (!isAnnotatedNonEmptyReaction(text, utterances.get(worldName)))
		{
			++nNotAnnotated;
			System.out.println("\n[---Filtering Not Annotated or Annotated with Empty Reaction---]: " + text);
			System.out.println("[Time]: " + timestamp + ":" + timestamp.getTime() + "\n");
			return true;
		}
		return false;
	}


	/**
	 * Verifies if the given text is the last selected instruction
	 * that CL has given.
	 * @return True if 'text' is the last text message in the list 'messages'
	 * that should not be filtered.
	 */
	protected boolean isLastText(String text, List<String> messages, List<String> hardCoded, String worldName)
	{
		int to = messages.indexOf("Yay! You completed the task!");
		if (to == -1)
			return false;
		List<String> slice = messages.subList(0, to);
		for (int i=0; i<slice.size(); i++)
		{
			if (!isGeneratedByHand(slice.get(i), hardCoded) 
			&& isAnnotatedNonEmptyReaction(slice.get(i), utterances.get(worldName)))
				return false;
		}
		System.out.println("Is the Last Text: " + text);
		return true;
	}


	/**
	 * Retrieves a list containing all the text messages that are contained
	 * in the sublist [events[i], events[i+1], ..., events[events.length]]
	 */
	protected List<String> getTextMessages(int i, List<GameEvent> events)
	{
		List<String> textMessages = new ArrayList<String>();
		for (int j = i; j<events.size(); j++)
		{
			GameEvent event = events.get(j);
			Message message = event.message;
			if (message instanceof TextMessage)
				textMessages.add(cleanInstruction(((TextMessage) message).getText()));
		}
		return textMessages;
	}


	/**
	 * Simulates the given GIVE game log. For each instruction message
	 * on the log it calls the selector with the current context of
	 * the simulated game to reproduce the list of candidates that CL used
	 * in that moment. Then, it matches Utterances from the candidates list with
	 * log's instructions and creates UtteranceP objects for adding information
	 * about this specific instruction occurrence.
	 */
	public void analyseLog(World w, int gameId)
	{
		World world = new World();
		w.copyTo(world);
		List<World> worlds = new ArrayList<World>();
		List<World> initialStates = new ArrayList<World>();
		addNewWorld(world, worlds);

		Context context = new Context(ACLBuilder.getPlayerRegion(discretizer, world));
		Response reaction = new Response();
		
		String text = null;
		String newText = null;
		
		Date timestamp = null;
		Date newTimestamp = null;

		List<Occurrence> relatedOccurrences = new ArrayList<Occurrence>();
		List<String> hardCoded = getCLStaticMessages();
		
		List<GameEvent> events = connector.getGameEvents(gameId);

		for(int i = 0; i < events.size(); i++)
		{
			GameEvent event = events.get(i);
			Message message = event.message;

			// updates the world, the context and the reaction when an action is done
			if (message instanceof ActionMessage)
			{
				ACLBuilder.processActionMessage(world, (ActionMessage) message, context, reaction);
				addNewWorld(world, worlds);
			}

			else if (message instanceof StatusMessage)
			{
				if (ACLBuilder.processStatusMessage(world, (StatusMessage) message, context, reaction, discretizer))
				{
					System.out.println("[Time]: \n" + event.timestamp + ":" + event.timestamp.getTime());
					addNewWorld(world, worlds);
				}
			}

			else if (message instanceof TextMessage)
			{
				newText = cleanInstruction(((TextMessage) message).getText());
				timestamp = newTimestamp;
				newTimestamp = event.timestamp;
				List<Atom> actions = null;

				if (!initialStates.isEmpty())
				{
					// Computes the discretized reaction that the player executed
					World lastState = initialStates.get(initialStates.size()-1);
					actions = ACLBuilder.computeDiscretizedResponse(
							isLastText(text, getTextMessages(i, events), hardCoded, world.getName()), 
							planner, lastState, world); 
					if (actions != null)
						reaction.setDiscretizedActions(actions);
					
					if (!shouldBeFiltered(text, hardCoded, timestamp, world.getName()))
					{
						if (!processStatusTextMessage(text, initialStates, world,
								reaction, gameId, timestamp, relatedOccurrences))
						{
							System.out.println("Game " + gameId + " discarded");
							break;
						}
					}
				}
				initialStates.clear();
				initialStates.addAll(worlds);
				worlds.clear();
				addNewWorld(world, worlds);
				if (actions != null && !actions.isEmpty())
				{
					for (Occurrence o : relatedOccurrences)
						o.addRelatedOccurences(relatedOccurrences);
					relatedOccurrences.clear();
				}
				reaction = new Response();
				text = newText;
				++nUtts;
			}
		}

		// Last text message.
		if (!initialStates.isEmpty() && reaction != null)
		{
			// Computes the discretized reaction that the player executed
			World lastState = initialStates.get(initialStates.size()-1);
			List<Atom> actions = ACLBuilder.computeDiscretizedResponse(false, planner, lastState, world);
			if (actions != null)
				reaction.setDiscretizedActions(actions);
			if (text != null  && newTimestamp != null
			&& !shouldBeFiltered(text, hardCoded, newTimestamp, world.getName()))
			{
				processStatusTextMessage(text, initialStates, world, 
						reaction, gameId, newTimestamp, relatedOccurrences);
			}
		}
		for (Occurrence o : relatedOccurrences)
			o.addRelatedOccurences(relatedOccurrences);

	}


	/**
	 * Uses the planner to recompute the first fragment of the raw plan. 
	 * @return A list of atoms. A optimized plan in the sense that its 
	 * move actions until the first manipulate or take action are the optims.
	 */
	protected static List<Atom> getOptimizedPlan(PlannerInterface planner, World world, List<Formula> goals) 
	{
		List<Atom> optimizedMoves = new ArrayList<Atom>();
		try
		{
			List<Atom> optimizedPlan = planner.computePlan(world, goals);
			
			if (optimizedPlan!=null && !optimizedPlan.isEmpty())
			{
				List<Atom> actions = SelectorGiveServer.getFutureActions(optimizedPlan);
				Atom nextAtom = actions.get(0);
				//System.out.println("Next manipulate: "+nextAtom);
				Action nextAction = world.findAction(nextAtom);
				//System.out.println("Next manipulate: "+nextAction);
				
				World goalState = new World();
				world.copyTo(goalState);
				nextAction.apply(nextAtom, goalState);
				
				List<Formula> goalFormula = ACLBuilder.getGoalsWithoutPos(goalState);
				
			    List<Atom> optimizedUntilAction = planner.computePlan(world, goalFormula);
			    //System.out.println("OPTIMIZED until action: "+optimizedUntilAction);
			    
			    optimizedMoves = SelectorGiveServer.getFutureMoveUntilAction(optimizedUntilAction);
			    List<Atom> oldMoves = SelectorGiveServer.getFutureMoveUntilAction(optimizedPlan);
			    optimizedPlan.removeAll(oldMoves);
			    optimizedMoves.addAll(optimizedPlan);
			    //System.out.println("OPTIMIZED plan: "+optimizedMoves);
			}
			else
				System.out.println("Null plan");
			
		}
		catch (IOException e)
		{
			e.printStackTrace();
		}
		return optimizedMoves;
		
	}
	
	
	/**
	 * De-serialize the given file into a Map<String, Set<Utterance>> object.
	 * @param file
	 */
	@SuppressWarnings("unchecked")
	public static Map<String, Set<Utterance>> load(File file)
	{
		Map<String, Set<Utterance>> utterances = null;
		FileInputStream fis = null;
		GZIPInputStream gis = null;
		ObjectInputStream in = null;
		try
		{
			fis = new FileInputStream(file);
			gis = new GZIPInputStream(fis);
			in = new ObjectInputStream(gis);
			utterances = (Map<String, Set<Utterance>>) in.readObject();
			in.close();
		}
		catch (InvalidClassException ex)
		{
			System.err.println("Exception: " + ex.getMessage());
			System.err.println("Unable to read " + file + " : probably the Utterance class has changed since");
			System.err.println("Try to regenerate the file with UtteranceBuilder");
			System.exit(-1);
		}
		catch (IOException ex)
		{

			ex.printStackTrace();
		}
		catch (ClassNotFoundException ex)
		{
			ex.printStackTrace();
		}
		return utterances;
	}


} // End of the class
